Imports System.Linq

''' <summary>
''' Class to provide importing of LSPC OUT files; FileName is full path to last active OUT file; TableName will include list of all filenames (w/o extension) in same directory (analogous to station ID)
''' </summary>
Public Class clsDataLspcOut
    Inherits clsData

    Protected sr As IO.StreamReader
    Dim WithEvents csv As LumenWorks.Framework.IO.Csv.CsvReader
    Protected mFileLength As Integer
    Protected mLastLineRead As String
    Protected cBufferSize As Integer = 100000

    ''' <summary>
    ''' Instantiate clsDataLSPC
    ''' </summary>
    ''' <param name="_FileName">Name of text file to read</param>
    Public Overrides Sub Initialize(ByVal _FileName As String)
        MyBase.Initialize(_FileName)
        mDataType = enumDataType.LspcOut
        Open(_FileName)
        ''will have one .OUT file as filename from which to get parameters; also will get list of all OUT files for ActiveTable (without .OUT extension) so can use for StationIDs
        mStationIDs.Clear()
        For Each fn As String In My.Computer.FileSystem.GetFiles(IO.Path.GetDirectoryName(_FileName), FileIO.SearchOption.SearchTopLevelOnly, "*" & IO.Path.GetExtension(_FileName))
            mStationIDs.Add(IO.Path.GetFileNameWithoutExtension(fn))
        Next
        mStationIDs.Sort()
        'mActiveStationID = mStationIDs(0)
        mActiveStationID = IO.Path.GetFileNameWithoutExtension(_FileName) 'default to selected file
    End Sub

    ''' <summary>
    ''' Set or get the currently selected table (the OUT file in the selected directory)
    ''' </summary>
    ''' <value></value>
    ''' <returns></returns>
    ''' <remarks>Set the fieldnames, get the number or records and leave the connection open to start reading</remarks>
    Public Overrides Property ActiveTable() As String
        Get
            Return MyBase.ActiveTable
        End Get
        Set(ByVal value As String)
            'Debug.Assert(False, "Called LSPC ActiveTable!")
            MyBase.ActiveTable = value
            'Open(String.Format("{0}\{1}.out", IO.Path.GetDirectoryName(mFilename), mActiveTable))
        End Set
    End Property

    ''' <summary>
    ''' Set or get the currently selected station ID (the OUT file in the selected directory)
    ''' </summary>
    ''' <value></value>
    ''' <returns></returns>
    ''' <remarks>Set the fieldnames, get the number or records and leave the connection open to start reading</remarks>
    Public Overrides Property ActiveStationID() As String
        Get
            Return MyBase.ActiveStationID
        End Get
        Set(ByVal value As String)
            MyBase.ActiveStationID = value
            Open(String.Format("{0}\{1}{2}", IO.Path.GetDirectoryName(mFilename), mActiveStationID, IO.Path.GetExtension(mFilename)))
        End Set
    End Property

    '''' <summary>
    '''' Determine index of named field (case insensitive) (assumes that fieldname is already parsed; will parse fieldnames using spaces since include parm name and units)
    '''' </summary>
    '''' <param name="FieldName">Name of field to check</param>
    '''' <returns>Index if found, -1 if not found</returns>
    'Public Overrides Function FieldIndex(ByVal FieldName As String) As Integer
    '    For i As Integer = 0 To mFieldNames.Count - 1
    '        If mFieldNames(i).Split(" ")(0).Equals(FieldName, StringComparison.OrdinalIgnoreCase) Then Return i
    '    Next
    '    Return -1
    'End Function

    ''' <summary>
    ''' Given a fieldname, return the parsed value representing the PCode (used primarily of LSPC/EFCD import)
    ''' </summary>
    ''' <param name="FieldName">Field name</param>
    Public Overrides Function PCodeFromField(ByVal FieldName As String) As String
        Return Strings.Left(FieldName.Split(IIf(FileName.ToLower.EndsWith(".air"), ",", " "))(0), 10)
    End Function

    ''' <summary>
    ''' Instantiate and get list of parameter names (note that entire line after TT will be set as field name, including parameter name, description, and units)
    ''' </summary>
    ''' <param name="_FileName">Name of LSPC OUT file to read</param>
    Friend Overridable Sub Open(ByVal _FileName As String)
        If Not My.Computer.FileSystem.FileExists(_FileName) Then
            Throw New ApplicationException("The LSPC file was not found: " & _FileName)
            Exit Sub
        End If
        mFilename = _FileName
        mRecordNum = -1
        If sr IsNot Nothing Then sr.Close() : sr.Dispose()
        sr = New IO.StreamReader(_FileName, System.Text.Encoding.ASCII, False, cBufferSize)
        mFileLength = sr.BaseStream.Length

        csv = New LumenWorks.Framework.IO.Csv.CsvReader(sr, False, vbTab, """", "\", "T", cBufferSize)

        'get fieldnames from header of first OUT file found (or leave empty)
        mFieldNames.Clear()
        mPCodes.Clear()

        mLastLineRead = ""
        Do
            mLastLineRead = sr.ReadLine
            If sr.EndOfStream Then
                Throw New ApplicationException("Unable to find constituent names in " & _FileName)
                sr.close()
                sr.dispose()
                Exit Sub
            End If
        Loop Until mLastLineRead.Trim = "TT Label"
        mFieldNames.Add("Station_ID")
        mFieldNames.Add("Date_Time")
        mLastLineRead = sr.ReadLine
        Do While mLastLineRead.Trim <> "TT"
            Dim PCode As String = mLastLineRead.Substring(3) 'strip off "TT "
            mFieldNames.Add(PCode)
            mPCodes.Add(PCodeFromField(PCode)) 'strip off pcode name and units
            mLastLineRead = sr.ReadLine
        Loop
        'Do While mLastLineRead.StartsWith("TT")
        '    mLastLineRead = sr.ReadLine
        'Loop
    End Sub

    ''' <summary>
    ''' Call this when need to write to already open file
    ''' </summary>
    Public Overrides Sub Close()
        If sr IsNot Nothing Then sr.Close()
    End Sub

    ''' <summary>
    ''' Read next line from file into memory
    ''' </summary>
    ''' <returns>True if successful</returns>
    Public Overrides Function Read() As Boolean
        Try
            If Not MyBase.Read() Then Return False

            'if called by clsDataLspcAir, don't want to do more
            If csv Is Nothing Then Return True

            If csv.EndOfStream Then 'go to next station ID
                If mActiveStationIDSet Then
                    Return False 'only read one stationid
                Else
                    Dim idx As Integer = mStationIDs.IndexOf(mActiveStationID)
                    If idx = mStationIDs.Count - 1 Then
                        Return False
                    Else
                        ActiveStationID = mStationIDs(idx + 1)
                        mActiveStationIDSet = False 'to keep it going
                    End If
                End If
            End If

            Try
                With csv
                    .DefaultParseErrorAction = LumenWorks.Framework.IO.Csv.ParseErrorAction.RaiseEvent
                    If .ReadNextRecord() Then
                        mItems.Clear()
                        mItems.Add(.Item(0).Trim)
                        Dim hr As Integer = .Item(4)
                        Dim mn As Integer = .Item(5)
                        If hr = 24 Then
                            'must convert to date (is slow) then add a day
                            Dim dt As Date = New Date(.Item(1), .Item(2), .Item(3), 0, mn, 0)
                            dt = dt.AddDays(1)
                            mItems.Add(dt.ToString("MM/dd/yyyy HH:mm"))
                        Else
                            mItems.Add(String.Format("{0}/{1}/{2} {3:00}:{4:00}", .Item(2), .Item(3), .Item(1), hr, mn)) 'date as string
                        End If
                        For i As Integer = 7 To .FieldCount - 1
                            mItems.Add(.Item(i).Trim) 'all other pcodes=2..n
                        Next
                        Return True
                    Else
                        Return False
                    End If
                End With
            Catch ex As Exception
                'will capture problems in event handler
                Return False
            End Try

            Return True
        Catch ex As Exception
            ErrorMsg(, ex)
            Return False
        End Try

    End Function

    ''' <summary>
    ''' Read next line from file into memory (faster method that only reads date and single column
    ''' </summary>
    ''' <returns>True if successful</returns>
    Friend Overridable Function ReadItem(ByVal ItemNum As Integer, ByRef DateTime As Date, ByRef Result As Single) As Boolean
        Try
            If Not MyBase.Read() Then Return False
            If csv.EndOfStream Then 'go to next station ID
                If mActiveStationIDSet Then
                    Return False 'only read one stationid
                Else
                    Dim idx As Integer = mStationIDs.IndexOf(mActiveStationID)
                    If idx = mStationIDs.Count - 1 Then
                        Return False
                    Else
                        ActiveStationID = mStationIDs(idx + 1)
                        mActiveStationIDSet = False 'to keep it going
                    End If
                End If
            End If

            Try
                With csv
                    .DefaultParseErrorAction = LumenWorks.Framework.IO.Csv.ParseErrorAction.RaiseEvent
                    If .ReadNextRecord() Then
                        mItems.Clear()
                        Dim hr As Integer = .Item(4)
                        If hr = 24 Then 'must add a day
                            DateTime = New Date(.Item(1), .Item(2), .Item(3), 0, .Item(5), 0)
                            DateTime = DateTime.AddDays(1)
                        Else
                            DateTime = New Date(.Item(1), .Item(2), .Item(3), hr, .Item(5), 0)
                        End If
                        Result = .Item(ItemNum + 7) 'there is an extra tab between minutes and first value
                        Return True
                    Else
                        Return False
                    End If
                End With
            Catch ex As Exception
                'will capture problems in event handler
                Return False
            End Try
            Return True
        Catch ex As Exception
            ErrorMsg(, ex)
            Return False
        End Try

    End Function

    ''' <summary>
    ''' Get data table containing time series data for current station id and all pcodes
    ''' </summary>
    Public Overloads Function GetDataTable(ByVal StationID As String) As DataTable
        Dim dt As DataTable = MyBase.SetupDataTable(enumDataTableColumns.All)
        ActiveStationID = StationID
        While Read()
            Dim dr As DataRow = dt.NewRow
            For i As Integer = 0 To NumItems - 1
                dr.Item(i) = Items(i)
            Next
            dt.Rows.Add(dr)
        End While
        Return dt
    End Function

    ''' <summary>
    ''' Get data table containing time series data for a specified station id and pcode
    ''' Table will contain date-time field and result field
    ''' </summary>
    Public Overloads Function GetDataTable(ByVal StationID As String, ByVal PCode As String) As DataTable
        Dim itemNum As Integer = -1
        If mPCodes.Contains(PCode) Then
            itemNum = mPCodes.IndexOf(PCode)
        Else
            Throw New ApplicationException("LSPC file does not contain the column: " & PCode)
            Return Nothing
        End If
        ActiveStationID = StationID
        Dim dt As DataTable = SetupDataTable(enumDataTableColumns.Date_Result)
        Dim DateTime As Date, Result As Single
        While ReadItem(itemNum, DateTime, Result) 'faster method to return only date/time and single column (it's only a little faster)
            dt.Rows.Add(DateTime, Result)
        End While
        Return dt
    End Function

    ''' <summary>
    ''' Determine min/max values for each pcodes regardless of station id
    ''' </summary>
    ''' <remarks></remarks>
    Protected Overrides Sub GetMinMaxDates()
        Rewind()
        Dim dummy As Single = 0, dt As Date = Nothing
        'get first date-time
        ReadItem(0, mMinDate, dummy)
        'loop for one out file through all lines and return last date-time read
        While ReadItem(0, dt, dummy)
            mMaxDate = dt
        End While
    End Sub

    ' ''' <summary>
    ' ''' Get aggregate value for all data in given time range for station id and pcode
    ' ''' If requested station ID or PCode is not found, will throw exception and return double.minvalue
    ' ''' </summary>
    ' ''' <param name="StationID">Desired Station ID</param>
    ' ''' <param name="PCode">Desired PCode</param>
    ' ''' <param name="MinDate">Minimum date to include</param>
    ' ''' <param name="MaxDate">Maximum date to include</param>
    'Public Overrides Function GetResult(StationID As String, PCode As String, MinDate As Date, MaxDate As Date, AggregateFunction As enumAggregateFunction) As Single
    '    Static LastPCode As String = ""

    '    'use thread-safe concurrent dictionary for parallel processing
    '    Static dictTables As Concurrent.ConcurrentDictionary(Of String, DataTable) = Nothing

    '    If PCode <> LastPCode Then

    '        'need to acquire data into datatable for fast extraction; data table will contain Date_Time, Station_ID, and Values
    '        'if later call with different Station_ID, Dates, or Aggregation, will just filter & compute the table; if different PCode will need to reacquire

    '        'use parallel processing to fill the table faster

    '        'use thread-safe concurrent dictionary for parallel processing
    '        dictTables = New Concurrent.ConcurrentDictionary(Of String, DataTable)

    '        Threading.Tasks.Parallel.For(0, mStationIDs.Count - 1, Sub(i)
    '                                                                   Dim staID As String = mStationIDs(i)
    '                                                                   Dim ds As WRDB.DataSource.clsData = WRDB.DataSource.clsData.Instantiate(mFilename)
    '                                                                   Dim dt As DataTable = ds.GetDataTable(staID, PCode)
    '                                                                   dictTables(staID) = dt
    '                                                                   RaiseProgress(i, mStationIDs.Count - 1, "Compiling data: " & staID)
    '                                                               End Sub)
    '    End If

    '    'I've assembled time series for this pcode for all stations, return aggregate value within date range for this particular station
    '    Dim dtData As DataTable = dictTables(StationID)

    '    Dim rowfilter As String = String.Format("Date_Time>=#{0}# AND Date_Time<#{1}#", MinDate, MaxDate)

    '    Dim value As Double = -1

    '    If MinDate = MaxDate Then
    '        Select Case AggregateFunction
    '            Case enumAggregateFunction.Count : Return 1
    '            Case Else : Return TestNull(dtData.Select(rowfilter)(0)(1), -1)
    '        End Select
    '    Else
    '        Dim aggFunc As String = ""
    '        Select Case AggregateFunction
    '            Case enumAggregateFunction.Count : aggFunc = "Count"
    '            Case enumAggregateFunction.Minimum : aggFunc = "Min"
    '            Case enumAggregateFunction.Maximum : aggFunc = "Max"
    '            Case enumAggregateFunction.Average : aggFunc = "Avg"
    '        End Select
    '        Return TestNull(dtData.Compute(String.Format("{0}([{1}])", aggFunc, StationID), rowfilter), -1)
    '    End If

    'End Function


    ''' <summary>
    ''' Get data table containing time series data for current station id and all pcodes
    ''' </summary>
    Public Overrides Function GetDataTable() As DataTable
        Dim dt As DataTable = MyBase.GetDataTable()
        While Read()
            Dim dr As DataRow = dt.NewRow
            For i As Integer = 0 To NumItems - 1
                dr.Item(i) = Items(i)
            Next
            dt.Rows.Add(dr)
        End While
        Return dt
    End Function

    Public Overrides ReadOnly Property MinDate() As Date
        Get
            Rewind()
            If Read() Then
                Return CDate(Items(1))
            Else
                Return MyBase.MinDate
            End If
        End Get
    End Property

    Public Overrides ReadOnly Property MaxDate() As Date
        Get
            Rewind()
            If Read() Then
                'get line length
                'Dim len As Integer = mLastLineRead.Length
                'reposition near the end of the file
                'sr.BaseStream.Seek(-Len() * 2, IO.SeekOrigin.End)
                'sr.DiscardBufferedData()
                Dim lastdate As Date
                'While Not sr.EndOfStream
                'sr.ReadLine() 'discard this one
                'mLastLineRead = sr.ReadLine
                'Dim ar() As String = mLastLineRead.Split(vbTab)
                'lastdate = New Date(ar(1), ar(2), ar(3), ar(4), ar(5), 0)
                'End While
                With csv
                    While .ReadNextRecord
                        lastdate = MyCDate(String.Format("{0}/{1}/{2} {3:00}:{4:00}", .Item(2), .Item(3), .Item(1), .Item(4), .Item(5)))
                    End While
                End With

                Return lastdate
            Else
                Return MyBase.MaxDate
            End If
        End Get
    End Property

    Public Overrides Sub Rewind()
        MyBase.Rewind()
        Open(mFilename)
    End Sub

    Public Overrides Function PercentComplete() As Double
        Return CInt(sr.BaseStream.Position * 100.0 / mFileLength)
    End Function


    Private disposed As Boolean = False

    Protected Overrides Sub Dispose(ByVal disposing As Boolean)
        If Not Me.disposed Then
            If disposing Then
                sr.Close()
                sr.Dispose()
                sr = Nothing
            End If
        End If
        Me.disposed = True
    End Sub

End Class
